package txt;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.hutool.core.io.FileUtil;
import com.panovue.dataextraction.reader.AbstractReader;
import com.panovue.dataextraction.reader.Reader;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;

@Slf4j
public class KrishnamurtiReader extends AbstractReader {

    public KrishnamurtiReader() {
        super("krishnamurti", true);
    }

    public KrishnamurtiReader(String key) {
        super("krishnamurti", true);
        this.key = key;

        baseUrl = "https://www.hservers.org/kcollection/Krishnamurti%20Complete%20Works/";
        baseDir += key + File.separator;

        // 设置线程数
        setThreads(5);
        addSeedAndReturn(baseUrl).type("list");

        getConf().setExecuteInterval(3000);

        buildDriver(true);

        // 开始爬取
        try {
            start(20);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public void visit(Page page, CrawlDatums next) {

    }

    @MatchType(types = "list")
    public void list(Page page, CrawlDatums next) {
        log.info("url {}", page.url());
        Elements elements = page.select("a");
        for (Element element : elements) {
            String href = element.attr("href");
            String text = href.substring(0,href.indexOf("/")+1);
            next.addAndReturn(baseUrl + href).type("book").meta("bookName", text);
        }

    }

    @MatchType(types = "book")
    public void book(Page page, CrawlDatums next) {
        log.info("url {}", page.url());
        String bookName = page.meta("bookName");
        File file = new File(baseDir + bookName + "chapter.html");
        FileUtil.writeUtf8String(page.html(),file);
        Elements elements = page.select("a");
        for (Element element : elements) {
            String href = element.attr("href");
            next.addAndReturn(baseUrl+ bookName + href).type("chapter").meta("bookName", bookName).meta("chapterName", href);
        }
    }

    @MatchType(types = "chapter")
    public void chapter(Page page, CrawlDatums next) {
        String url = page.url();
        log.info("url {}", url);
        String bookName = page.meta("bookName");
        String chapterName = page.meta("chapterName");
        File file = new File(baseDir + bookName + chapterName);
        if (!file.exists()) {
            FileUtil.writeUtf8String(page.html(),file);
        }
    }

    @MatchCode(codes = {301, 302, 307})
    public void retry(Page page, CrawlDatums next) {
        CrawlDatum crawlDatum = new CrawlDatum();
        crawlDatum.url(page.location());
        crawlDatum.type(page.matchType("list") ? "list" : "details");
        crawlDatum.key(crawlDatum.getExecuteTime() + page.url());
        next.addAndReturn(crawlDatum);
    }

    @Override
    public void afterStop() {
        if (driver != null) {
            driver.quit();
        }
        log.info("krishnamurti 数据爬取完成");
    }

    @Override
    public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
        super.execute(datum, next);
        log.info("url {} code {}", datum.url(), datum.code());
    }

    @Override
    public Reader buildReader(String key) {
        return new KrishnamurtiReader(key);
    }

    public static void main(String[] args) {
        new KrishnamurtiReader().buildReader("krishnamurti");
    }
}
